#author : kqy
#date : 2010-04-29

require 'rubygems'
require 'open-uri'
require 'logger'
require 'net/http'
require 'spider/agent'

module Spider
  def self.included(base)
    base.extend ClassMethods
  end

  module ClassMethods
    def regexp_crawler(options)
      @crawlers ||= []
      @crawlers = Agent.new(options)
    end

    def start_crawl
      @crawlers.each do |crawler|
        crawler.start
      end
    end
  end
end
